{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import steward as st\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import xgboost\n",
    "from sklearn.metrics import roc_auc_score\n",
    "%matplotlib inline\n",
    "from src import build\n",
    "from src import test, config\n",
    "from src.feature_cols import to_drop\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "build.build_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list = [\n",
    "\n",
    "    'basic_preprocess/cont_{}',\n",
    "    'basic_preprocess/conc_{}',\n",
    "    \n",
    "    \n",
    "    'feature/rank1_{}',\n",
    "    'feature/rank2_{}',\n",
    "    'feature/rank3_{}',\n",
    "    \n",
    "    'feature/history1_{}',\n",
    "    'feature/history2_{}',\n",
    "    'feature/history3_{}',\n",
    "\n",
    "    'feature/ordnum_{}',\n",
    "    'feature/room1_{}',\n",
    "    'feature/basicroom1_{}',\n",
    "    'feature/orderroom1_{}',\n",
    "    \n",
    "    'feature/rank_history1_{}',\n",
    "    'feature/rank_history2_{}',\n",
    "    'feature/rank_history3_{}',\n",
    "    \n",
    "    'feature/room1small5_{}',\n",
    "    'feature/room1large5_{}',\n",
    "    'feature/orderroom1small5_{}',\n",
    "    'feature/orderroom1large5_{}',\n",
    "\n",
    "    'feature/custom1_{}',\n",
    "    'feature/roomcount_date_{}',\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = 'train_all'\n",
    "feature_list = [feature_name.format(dataset) for feature_name in feature_list]\n",
    "N_rows = 7724875 # for train\n",
    "# N_rows = 7448647 # for test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t = st.LoadInstance('test/real_all_without_head_30W')\n",
    "t()\n",
    "im = t.df['im']\n",
    "result = t.df['result']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame(result, columns=['n', 'iter_n', 'score'])\n",
    "top_cols = im.head(395).index.tolist()\n",
    "select_cols = top_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N_cols = len(select_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = h5py.File('data/feature_{}.hdf5'.format(dataset), 'w')\n",
    "dset = f.create_dataset(\"default\", (N_rows, N_cols), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cols = []\n",
    "cols_saved = 0\n",
    "for i, feature_name in enumerate(feature_list):\n",
    "    x = st.get_instance(feature_name)\n",
    "    df = x.load()\n",
    "    to_save_cols = df.columns[df.columns.isin(select_cols)]\n",
    "    dset[:, cols_saved:cols_saved+len(to_save_cols)] = df[to_save_cols].values\n",
    "    cols_saved = cols_saved+len(to_save_cols)\n",
    "    cols.extend(to_save_cols)\n",
    "    del df\n",
    "    del st.default.session.df_dict[x]\n",
    "    print('saved:', feature_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
